#!/usr/local/bin/python
# -*- encoding:utf-8 -*-

import numpy as np;
import scipy.sparse as sp
from scipy import linalg
import time
import threading
import os
from scipy.sparse.linalg import eigsh,inv
from multiprocessing import Process, Manager
import sys
import utils
from whoosh.index import open_dir
from whoosh.fields import *
from whoosh.scoring import *
from whoosh.matching import *
import jieba.analyse
import jieba
from whoosh.qparser import QueryParser

'''This script analysis the tagset, and find the word in document coocurrence with tagset.
Hopefully this synonyms will help to imporve the retrieval effectiveness'''



config = utils.get_config();

analyzer = jieba.analyse.ChineseAnalyzer();
f = formats.Frequency();
schema = Schema(appid = ID(stored=True), content=TEXT(stored=True, analyzer= analyzer, vector = f));
ix = open_dir(config.get('bm25', 'BackupIndex'));
reader = ix.reader();
searcher = ix.searcher();

parser = QueryParser('content', schema);
weightingM = BM25F();
dictMap = {};
d = open(config.get('rmls', 'dictionary'), 'r');
index = 0;
for line in d:
    dictMap[line.strip('\n').split('\t')[0].decode('utf-8')] = index;
    index += 1;
d.close();
MAX = len(dictMap);
print MAX;

def getDocumentFeature(appID):
    di = {}
    docnum = searcher.document_number(appid= appID);
    if not docnum:
        return None;
    if not reader.has_vector(docnum, 'content'):
        return None;
    vec = reader.vector_as('frequency', docnum, 'content');
    for v in vec:
        term = v[0];
        tf = int(v[1]);
        score = weightingM.score(searcher, 'content', term, docnum, tf);
        di[term] = score
    return di;  


def format(di):
    if di == None:
        return None;
    if len(di) == 0:
        return None;
    for key, value in di.items():
        if not dictMap.has_key(key):
            di.pop(key);
    return di;


tagsDict = {};
'''This tagsDict stores the tags: words in bm25 weights'''


def updateWeights(d, di):
    if di == None:
        return;
    if len(di) == 0:
        return;
    for key, value in di.items():
        if d.has_key(key):
            d[key] += value;
        else:
            d[key] = value;

def loadTagSet(tagset):
    tagsetFile = open(tagset, 'r');
    for line in tagsetFile:
        li = line.strip().split('|');
        appID = li[0];
        documentFeature = format(getDocumentFeature(appID));
        for i in range(1, len(li)):
            tag = li[i];
            if tagsDict.has_key(tag):
                updateWeights(tagsDict[tag], documentFeature);
            else:
                tagsDict[tag] = {};
                updateWeights(tagsDict[tag], documentFeature);


def findIndex(s):
    if dictMap.has_key(s):
        return dictMap[s];
    else:
        print 'WRONG', s;
        return -1;

def findPhraseIndex(s):
    di = {}
    tli = analyzer(s.decode('utf-8'));
    for t in tli:
        term = t.text;
        if di.has_key(term):
            di[term] += 1;
        else:
            di[term] = 1;
    res = [];
    for key, value in di.items():
        if dictMap.has_key(key):
            res.append(dictMap[key]);
        else:
            pass;
    return res;

def outputResult(result1, result2):
    TOPK = 3;
    resultFile1 = open(result1, 'w');
    resultFile2 = open(result2, 'w');
    print len(tagsDict);
    for key, value in tagsDict.items():
        li = [(y, x) for x,y in value.items()];
        li.sort(reverse=True);
        s = 0.0;
        for i in range(min(len(li), TOPK)): 
            s += li[i][0];
            if key == li[i][1].encode('utf-8'):
                continue;
            resultFile1.write("%s\t%s\t%f\n" % (key, li[i][1].encode('utf-8'), li[i][0]));

        for i in findPhraseIndex(key):
            for j in range(min(len(li), TOPK)):
                if i == findIndex(li[j][1]):
                    continue;
                resultFile2.write("%d\t%d\t%f\n" % (i, findIndex(li[j][1]), li[j][0]/s));
    resultFile1.close();
    resultFile2.close();

if __name__ == '__main__':

    tagset = sys.argv[1];
    result1 = sys.argv[2];
    result2 = sys.argv[3];
    loadTagSet(tagset);
    outputResult(result1, result2);
